# -*- coding: utf-8 -*-

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# Author;xiaobudian

from multiprocessing import Pool
import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
import pymongo
import os
import sys
import traceback

ua = UserAgent()
headers1 = {'User-Agent': 'ua.ramdom'}
Mongo_Url = 'localhost'
Mongo_DB = 'Lianjia'
Mongo_TABLE = 'sub_area'
client = pymongo.MongoClient(Mongo_Url)
db = client[Mongo_DB]

def get_all_areas():  # 生成url
    url = 'https://bj.lianjia.com/ershoufang/'
    res = requests.get(url, 'lxml', headers=headers1)
    if res.status_code == 200:
        soup = BeautifulSoup(res.text, 'lxml')
        areas = soup.find(class_='position').find_all('a')
        for area in areas:
            area_url = area['href']
            if 'https' not in area_url and '/ershoufang/' != area_url and '/ditiefang/' != area_url:
                yield "https://bj.lianjia.com" + area_url


def get_all_sub_areas(area_url):
    print(area_url)
    res = requests.get(area_url, 'lxml', headers=headers1)
    if res.status_code == 200:
        soup = BeautifulSoup(res.text, 'lxml')
        for sub_area in soup.find('div',{"data-role":"ershoufang"}).find_all('a'):   
            if 'https' not in sub_area['href']:
                info = {}
                info['sub_area_url'] = sub_area['href'][11:]
                info['name'] = sub_area.text
                yield info
            

def update_to_MongoDB(info): 
    print(info)
    db[Mongo_TABLE].update({'sub_area_url': info['sub_area_url']}, {'$set': info}, True)

if __name__ == '__main__':

    with Pool(os.cpu_count()) as pool:
        for area_url in get_all_areas():
            pool.map(update_to_MongoDB, [sub_area_info for sub_area_info in get_all_sub_areas(area_url)])
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            
            